#include"DictProducer.h"
#include <sys/types.h>
#include <dirent.h>
#include<iostream>
#include<fstream>
#include<sstream>
#include<algorithm>
#include<nlohmann/json.hpp>

using std::cout;
using std::ifstream;
using std::istringstream;
using std::ofstream;
using std::count;
using std::cerr;

int getbytenum_utf8(char byte)
{
    int bytenum=0;
    for(int i =0 ; i<6;++i)
    {
        if(byte&(1<<(7-i)))
        {
            ++bytenum;
        }
        else break;
    }
    return bytenum = bytenum == 0? 1: bytenum;
}
//词典类构造
DictProducer::DictProducer(string dirpath,SplitTool* splitTool)
:_cuttor(splitTool)
{
    DIR* dir= opendir(dirpath.c_str());
    if(!dir)
    {
        cout<<" open dir failed -DictProducer.cc -12\n";
    }
    struct dirent* content;
    while((content=readdir(dir))!=nullptr)
    {
        string filename= content->d_name;
        if(content->d_type==DT_REG)
        {
            _files.push_back(dirpath+"/"+filename);
        }
        else if(content->d_type==DT_DIR)
        {
            if(filename!="."&&filename!="..")
            {
                string cndirpath = dirpath+"/"+filename;
                DIR * cndir = opendir(cndirpath.c_str());
                struct dirent * cncontext;
                while((cncontext=readdir(cndir))!=nullptr)
                {
                    string cnname= cncontext->d_name;
                    if(cncontext->d_type==DT_REG)
                    {
                        _files.push_back(cndirpath+"/"+cnname);
                    }
                }
            }
        }
    }
}

//创建英文字典
void DictProducer::buildEndict()
{
    map<string,int> mapdict;
    set<string> stop;
    string buf;

    ifstream stopengifs("/home/sakura/all_file/search_engine/static/stop/stop_words_eng.txt");

    ifstream stopcnfis("/home/sakura/all_file/search_engine/static/stop/stop_words_zh.txt");
    string tmp;
    while(std::getline(stopengifs,tmp))
    {
        tmp.pop_back();
        stop.emplace(tmp);
    }
    while(std::getline(stopcnfis,tmp))
    {
        // cout<<"stopcn " <<tmp<<"\n";
        tmp.pop_back();
        stop.emplace(tmp);
    }
    
    for(auto& i: _files)
    {
        int times=std::count(i.begin(),i.end(),'/');

        const char* name = i.c_str();

        if(times >7)
        {
        // if(i.find("C3")!=string::npos)
        ifstream ifs(name);
        if(!ifs)
        {
            cout<< " cn file stream open failed\n";
        }

        ifs.seekg(0,std::ios::end);

        size_t len = ifs.tellg();

        ifs.seekg(0,std::ios::beg);

        char* context = new char[len+1]{0};
        cout<<"\ntt\n";
         ifs.read(context,len);
        //  cout<<context;
         for(size_t i=0;i<len;++i )
         {
            char ch=context[i];
            if(ch=='\n'||ch=='\r'||isblank(ch))
            {
            context[i]=' ';
            }
         }
           vector<string> cnwords= _cuttor->cut(context);
           for(auto& i: cnwords)
           {
            if(stop.find(i)==stop.end()&&getbytenum_utf8(i[0])==3)
            {
                ++mapdict[i];
            }
           }
         delete [] context;
        }
        else{

        ifstream ifs(name);
        ifs.seekg(0,std::ios::end);
        size_t len = ifs.tellg();
        ifs.seekg(0,std::ios::beg);
        char* context = new char[len+1]{0};

        ifs.read(context,len);

         for(size_t i=0;i<len;++i )
         {
            char ch=context[i];
            if(isalpha(ch))
            {
                context[i]=tolower(ch);
            }
            else 
            {
                if(ch!=' ')
                {
                    context[i]=' ';
                }
            }
         }
        //英文
        istringstream iss(context);

        string word;

        while(iss>>word)
        {
            if(stop.find(word)==stop.end())
            {
            ++mapdict[word];
            }
        }
        delete []context;
        }
    }
    for(auto& i:mapdict)
    {
        // cout<<i.first<<" "<<i.second<<" \n";
        _dict.push_back({i.first,i.second});
    }
    cout<<" vector dict create finish\n";
    //todo
}

void DictProducer::createIndex()
{
    // for(auto& i: _dict)
    for(size_t i= 0;i<_dict.size();++i)
    {
        string elemnt = _dict[i].first;
        for(size_t n=0;n< elemnt.size();++n)
        {
            //按照字节数进行子串读取
            int len= getbytenum_utf8(elemnt[n]);
            string elit = elemnt.substr(n,len);
            _index[elit].emplace(i);
            n +=(len-1);
        }
    }
    cout<<" create index finish\n";
}

//将字典存储到文件
void DictProducer::store()
{

    ofstream dictofs("/home/sakura/all_file/search_engine/data/dict.dat");
    if(!dictofs)
    {
        cerr<<" open dictofs failed\n";
    }
    
    ofstream indexofs("/home/sakura/all_file/search_engine/data/dictIndex.dat");
    if(!indexofs)
    {
        cerr<<" open indexofs failed\n";
    }

    nlohmann::json json_dict = _dict;
    nlohmann::json json_index = _index;

    dictofs<< json_dict.dump();
    indexofs<< json_index.dump();
}

//测试用
void DictProducer::showFiles() const
{
    //todo 
}

void DictProducer::showDict() const
{
    //todo
}
